#!/usr/bin/env python
# encoding: utf-8

"""
@version: 0.1
@author: Kouen
@license: Apache Licence
@email: jobkouen@outlook.com
@software: PyCharm Community Edition
@file: stock_spiders.py
@time: 2016/12/30 下午2:32
@简介:
    
"""
import os
import sys

sys.path.insert(0, os.getcwd()[:os.getcwd().find("AnalySignalSystem")]+"AnalySignalSystem")

from scrapy.contrib.spiders import Spider
from source.Report.Scrapy.sina.sina.items import SinaHQItem
import scrapy
import json


class SinaFinan(Spider):
    name = 'sina_hq'

    def start_requests(self):
        print("#"*10, "Start", "#"*10)
        with open("../../../../document/StockURL.json") as f:
            codes = json.load(f)

        for code in codes:
            item = SinaHQItem()
            item['code'] = code
            yield scrapy.Request(
                url="http://money.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/%s.phtml" % code,
                callback=self.stock_parse,
                meta={'item': item}
            )

    def stock_parse(self, response):
        item = response.meta['item']
        for year in response.xpath("//select[@name='year']/child::option/text()").extract():
            for month in [1, 2, 3, 4]:
                url = f"http://money.finance.sina.com.cn/corp/go.php/vMS_MarketHistory/stockid/{item['code']}.phtml?year={year}&jidu={month}"
                yield scrapy.Request(
                    url=url,
                    callback=self.stock_parse_data,
                    meta={'item': item}
                )

    def stock_parse_data(self, response):
        for i in response.xpath("//table[@id='FundHoldSharesTable']/child::tr")[1:]:
            item = SinaHQItem()
            item['code'] = response.meta['item']['code']
            row_d = i.xpath('td/div')[0]
            if 'href' in row_d.extract():
                item['date'] = row_d.xpath('a/text()').extract()[0].strip()
            else:
                item['date'] = row_d.xpath('text()').extract()[0].strip()

            values = list(map(lambda x: float(x),
                              filter(lambda x: x and '-' not in x,
                                     map(lambda x: x.strip(),
                                         i.xpath('td/div/text()').extract()))))
            item['open'] = values[0]
            item['max'] = values[1]
            item['close'] = values[2]
            item['min'] = values[3]
            item['tv'] = values[4]
            item['tm'] = values[5]

            yield item


if __name__ == '__main__':
    pass  # print os.getcwd()[:os.getcwd().find("AnalySignalSystem")]+"AnalySignalSystem"
